package testrank.affinity;

import static java.io.StreamTokenizer.TT_NUMBER;
import static java.io.StreamTokenizer.TT_WORD;

import java.io.BufferedReader;
import java.io.FileReader;
import java.io.IOException;
import java.io.Reader;
import java.io.StreamTokenizer;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Set;
import java.util.SortedMap;
import java.util.SortedSet;
import java.util.TreeMap;
import java.util.TreeSet;

import testrank.TestRankConstants;

/**
 * object that analyzes a file, with & without comments
 * 
 * @author Hagai Cibulski
 */
public class SourceFileAnalyzer {
	
	public static final String JAVA_KEYWORDS[] = { "abstract", "boolean",
		"break", "byte", "case", "catch", "char", "class", "const",
		"continue", "default", "do", "double", "else", "extends", "final",
		"finally", "float", "for", "goto", "if", "implements", "import",
		"instanceof", "int", "interface", "long", "native", "new",
		"package", "private", "protected", "public", "return", "short",
		"static", "strictfp", "super", "switch", "synchronized", "this",
		"throws", "transient", "try", "void", "volatile", "while" };
	
	/**
	 * TODO WordNet stop words
	 * The following words are normally used as function words, but also turn out to have rather odd (but correct) senses listed in WordNet: 
	 * I, a, an, as, at, by, he, his, me, or, thou, us, who
	 */
	public static final List<String> STOPWORDS = readStopwords();
	
	static SortedSet<String> appearingStopWords = new TreeSet<String>();

	private String filePath;
	private SortedMap<Integer, List<String>> words = new TreeMap<Integer, List<String>>();
	private SortedMap<Integer, List<String>> literals = new TreeMap<Integer, List<String>>();
	
	private static SortedMap<String, WordInfo> wordCount = new TreeMap<String, WordInfo>();
	private static Set<String> commonWords = null;
	
	private SortedSet<Integer> linesWithCode = new TreeSet<Integer>();

	public SourceFileAnalyzer(String filePath) throws IOException {
		this.filePath = SourcePathUtil.fix(filePath);

		findLinesWithCode(this.filePath);
		
		process(this.filePath);
	}
	
	private void findLinesWithCode(String filePath) throws IOException {
		filePath = SourcePathUtil.fix(filePath);
		Reader reader = new FileReader(filePath);
		
		MyStreamTokenizer tokenizer = new MyStreamTokenizer(reader);
		
		tokenizer.slashStarComments(true);
		tokenizer.slashSlashComments(true);
		tokenizer.eolIsSignificant(false);
		tokenizer.ordinaryChar('\'');
		
		int ttype = 0;
		while (ttype != StreamTokenizer.TT_EOF) {
			ttype = tokenizer.nextToken();
			linesWithCode.add(tokenizer.lineno());
		}
	}

	/**
	 * goes through the file and extract words and literals
	 * @throws IOException 
	 */
	private void process(String filePath) throws IOException {
		
		FileReader reader = new FileReader(filePath);
		MyStreamTokenizer tokenizer = new MyStreamTokenizer(reader);
		// need this so that comment lines are not ignored
		tokenizer.ordinaryChar('/');//comment out to ignore comments	
		tokenizer.eolIsSignificant(false);
		tokenizer.wordChars('\'', '\''); // apostrophe is part of word
		
		int ttype = 0;
		while (ttype != StreamTokenizer.TT_EOF) {
			ttype = tokenizer.nextToken();
			switch (ttype) {
			case TT_WORD:
				{
					String sval = tokenizer.sval;
					if (isStopword(sval))
						continue;
					
					sval = cleanApostrophe(sval);
					String[] parts = sval.split("\\.", -1);
					for (String sPart : parts) {
						if (isJavaIdentifier(sPart)) {
							List<String> idParts = getIdentiferParts(sPart);
							for (String idPart : idParts) {
				
								if (isJavaKeyword(sval)) {
									// keyword - ignore
								}
								else // comment this to store stopwords 					
								if (isStopword(idPart.toLowerCase())) {
									// stopword - ignore
								}
								else
								if (WordNetConnector.isWordNetWord(idPart)) {
									put(words, tokenizer.lineno(), idPart);	
								}
								else{
									put(literals, tokenizer.lineno(), idPart);
								}
							}
							
							if (idParts.size() > 1) {
								put(literals, tokenizer.lineno(), sPart);
							}
						}
					}
				}
				break;
			case '\"':
				String sval = tokenizer.sval;
				if (isStopword(sval))
					continue;

				String[] parts = sval.split("[/\\.]", -1);
				for (String sPart : parts) {
					sPart = cleanApostrophe(sPart);
					put(literals, tokenizer.lineno(), sPart);
				}
				
				break;
			case TT_NUMBER:
				// TODO integers are formated as floats; maybe something with parseNumbers
				// TODO need to use ordinaryChar or wordChars with 0..9, ., -
//				System.out.println("number:\t\t" + tokenizer.nval);
				put(literals, tokenizer.lineno(), String.valueOf(tokenizer.nval));
				break;
			default: // single character token
//				System.out.println(Character.valueOf((char)tokenizer.ttype));
				// here we get: ; * ( ) { } - = , @
				// TODO annotation appears here as '@'
			}
		}
	}

	private boolean put(SortedMap<Integer, List<String>> multimap, int key,
			String value) {
		List<String> list = multimap.get(key);
		if (list == null) {
			list = new ArrayList<String>();
			multimap.put(key, list);
		}
		
		updateTotalWordCount(value);
		
		return list.add(value);
	}

	private void updateTotalWordCount(String value) {
		WordInfo info = wordCount.get(value);
		if (info == null) {
			info = new WordInfo(value, 0); 
			wordCount.put(value, info);
		}
		info.add(1); 
	}

	/**
	 * creates a WordGroup with words and literals from start to end location of a method
	 */
	public WordGroup analyzeMethod(int startMethodLocation, int endMethodLocation) {
		int startLocation = findLastLineWithoutCodeBefore(startMethodLocation);
		WordGroup wordGroup = new WordGroup();
		{
			SortedMap<Integer, List<String>> subMap = getSubMap(words,
					startLocation, endMethodLocation);
			for (List<String> list : subMap.values()) {
				for (String s : list) {
					wordGroup.addWord(s);
				}
			}
		}
		{
			SortedMap<Integer, List<String>> subMap = getSubMap(literals,
					startLocation, endMethodLocation);
			for (List<String> list : subMap.values()) {
				for (String s : list) {
					wordGroup.addLiteral(s);
				}
			}
		}
		
//		wordGroup.addDocument();
		
		return wordGroup;
	}

	private SortedMap<Integer, List<String>> getSubMap(SortedMap<Integer, List<String>> map,
			int startLocation,
			int endMethodLocation) {
		SortedMap<Integer, List<String>> subMap = 
			endMethodLocation >= 0 ? //TODO why sometimes -1 (clue EndMarker.suite())
					map.subMap(startLocation,
				endMethodLocation + 1) ://TODO +2?
					map.tailMap(startLocation);
		return subMap;
	}
	
	private int findLastLineWithoutCodeBefore(int methodLocation) {
		SortedSet<Integer> codeLinesBeforeMethod = linesWithCode.headSet(methodLocation);
		while ( ! codeLinesBeforeMethod.isEmpty()) {
			int lastCodeLineBeforeMethod = codeLinesBeforeMethod.last();
			if (lastCodeLineBeforeMethod == methodLocation - 1) {
				codeLinesBeforeMethod = codeLinesBeforeMethod.headSet(methodLocation - 1);
				methodLocation--;
				// simple heuristic to find last code line unrelated to the method:
				// if last code line is the one before, it is most something like a single "public" or "static"  
			}
			else {
				return lastCodeLineBeforeMethod + 1;
			}
		}
		return 0;
	}

	public static List<String> getIdentiferParts(String sval) {
		String[] parts = sval.split("\\_", -1);
		List<String> idParts = new ArrayList<String>();
		for (String p : parts) {
			List<String> ccParts = splitCamelCase(p);
			idParts.addAll(ccParts);
		}
		return idParts;
	}

	// Returns true if s is a legal Java identifier.
	public static boolean isJavaIdentifier(String s) {
		if (s.length() == 0 || !Character.isJavaIdentifierStart(s.charAt(0))) {
			return false;
		}
		for (int i = 1; i < s.length(); i++) {
			if (!Character.isJavaIdentifierPart(s.charAt(i))) {
				return false;
			}
		}
		return true;
	}

	public static boolean isJavaKeyword(String s) {
		return java.util.Arrays.binarySearch(JAVA_KEYWORDS, s) >= 0;
	}
	
	public static boolean isStopword(String s) {
		boolean inStopWordsList = java.util.Collections.binarySearch(STOPWORDS, s) >= 0;
		if (inStopWordsList)
			appearingStopWords.add(s);
		
		return inStopWordsList; 
	}
	
	
	public static List<String> splitCamelCase(String s) {
		
		if (s.length() <= 1) {
			return Collections.singletonList(s);
		}
		
//		String s1 = s.charAt(0) + s.substring(1).replaceAll("([A-Z])"," $1");
//		s.replaceAll("([A-Z]([A-Z]*(?=$|[A-Z][a-z])|[a-z]*))", " $1");

		List<String> words = new ArrayList<String>();
		StringBuilder word = new StringBuilder();
		char c0 = s.charAt(0);
		word.append(c0);
		boolean lastCase = Character.isUpperCase(c0);
		for (int i = 1; i < s.length(); i++) {
			char c = s.charAt(i);
			boolean currCase = Character.isUpperCase(c);
			if (currCase == lastCase) {
				word.append(c);
			}
			else {
				if (lastCase && ! currCase) {
					if (word.length() == 1) {
						word.append(c);
					}
					else { 
						assert(word.length() > 1);
						
						words.add(word.substring(0, word.length() - 1));
						word  = new StringBuilder(word.charAt(word.length() - 1) + "" + c);
					}
				}
				else {
					assert ( ! lastCase && currCase);
					
					words.add(word.toString());
					word  = new StringBuilder(String.valueOf(c));
				}
			}
			lastCase = currCase;
		}
		if (word.length() > 0) {
			words.add(word.toString());
		}
	
		return words;
	}
	
	// TODO consider words versus literals, and test code versus application code
	// TODO extract Filter Strategy
	/*
	 * this implementation looks for 15% of words that are most frequent
	 */
	public static Set<String> getCommonWords() {
		if (commonWords == null) {
			commonWords = new TreeSet<String>();

			List<WordInfo> counts = new ArrayList<WordInfo>(); 
			for (WordInfo info  : wordCount.values()) {
				counts.add(info);
			}
			Collections.sort(counts);
			
			// look for (0.10) most frequent words
			int limit = (int)(0.10 * counts.size());
			for (int i = counts.size() - 1; i > counts.size() - limit; i--) {
				WordInfo info = counts.get(i);
				commonWords.add(info.getWord());
			}
			
//			System.out.println("--Word Counts (" + counts.size()+ " ): " + counts);
//			System.out.println("-- Common Words (" + commonWords.size() + "): "+ commonWords);
//			Set<String> commonWords1 = new TreeSet<String>(commonWords);
//			commonWords1.retainAll(STOPWORDS);
//			System.out.println("Common: "+commonWords.size() + ", STOP: " +STOPWORDS.size() + ", intersection: " + commonWords1.size());
//			System.out.println("\nActually appearing stopwords:\n" + appearingStopWords.toString().replaceAll(", ", "\n"));
		}

		return commonWords;
	}

	private static List<String> readStopwords() {
		List<String> stopwords = new ArrayList<String>();
		String aFile = TestRankConstants.CONF_DIR + "/" +
		"ranks.stopwords.txt";

		
		try {
			BufferedReader input = new BufferedReader(new FileReader(aFile));
			try {
				String line = null;
				while ((line = input.readLine()) != null) {
					stopwords.add(line);
					String clean = cleanApostrophe(line);
					if (!line.equals(clean))
						stopwords.add(clean);
//					System.out.println(line);
//					if (WordNetConnector.isWordNetWord(line)) {
//						System.out.println(line);
//					}
				}
			} finally {
				input.close();
			}
		} catch (IOException ex) {
			ex.printStackTrace();
		}
		Collections.sort(stopwords);
		
//		System.out.println("\nstopwords:\n" + stopwords.toString().replaceAll(", ", "\n"));
		
		return stopwords;
	}

	public static String cleanApostrophe(String s) {
		int index = s.indexOf('\'');
		if (index == -1)
			return s;
		
		if (index == s.length() - 2 && s.charAt(s.length() - 1) == 's')
			return s.substring(0, s.length() - 2);
		
		return s.replaceAll("\'", "");
	}
	
	public static void main(String[] args) throws IOException {
		String filePath = "src/testrank/affinity/SourceFileAnalyzer.java";
		new SourceFileAnalyzer(filePath);
		
//		System.out.println(cleanApostrophe("'simple"));		// simple
//		System.out.println(cleanApostrophe("quote'"));		// quote
//		System.out.println(cleanApostrophe("category's"));	// category
//		System.out.println(cleanApostrophe("boss's"));		// boss
//		System.out.println(cleanApostrophe("pens'"));		// pens
	}
}
